Notebook stuck?¶

Note that because of Jupyter and Plotly issues, sometimes the code may stuck at visualization. We recommend restart the kernels and try again to see if the issue is resolved.

In [1]:
import matplotlib.colors as colors
import matplotlib.pyplot as plt
import numpy as np
import plotly.graph_objects as go
from pathlib import Path
from PIL import Image
from scipy.spatial.transform import Rotation as R
import os

DATASET_ROOT = os.path.expanduser("~/dev/projectaria_tools/projectaria_tools_ase_data")  # Specify your own dataset path
SCENE_ID = 0  # Select a scene id

dataset_path = Path(DATASET_ROOT)
print("Chosen ASE data path: ", dataset_path)
print(f"Using Scene {SCENE_ID} for these examples")

scene_path = dataset_path / str(SCENE_ID)
Chosen ASE data path:  /Users/florian/dev/projectaria_tools/projectaria_tools_ase_data
Using Scene 0 for these examples

Part 1: Reading Scene Annotations and 3D Plotting¶

In [2]:
from code_snippets.readers import read_points_file, read_trajectory_file, read_language_file

# Load scene point cloud using read_points_file()
points_path = scene_path / "semidense_points.csv.gz"
points = read_points_file(points_path)

# Load a trajectory using read_trajectory_file() 
trajectory_path = scene_path / "trajectory.csv"
trajectory = read_trajectory_file(trajectory_path)

# Load a scene command language using read_language_file()
language_path = scene_path / "ase_scene_language.txt"
entities = read_language_file(language_path)
Loaded point cloud with 433426 points.
Loaded trajectory with 350 device poses.
Loaded scene commands with a total of 15 entities.
In [3]:
from code_snippets.interpreter import language_to_bboxes

# Interpret scene commands into 3D Boxes
entity_boxes = language_to_bboxes(entities)
Scene contains:
  wall: 8
  door: 3
  window: 4

Part 2: Loading and Plotting Images and Image Annotations¶

In [4]:
def random_bright_colormap(num_colors=1024):
    bright_colors = np.random.rand(num_colors, 3)
    bright_colors = (bright_colors + 1) / 2
    return colors.ListedColormap([c for c in bright_colors])

scene_path = dataset_path / str(SCENE_ID)

rgb_dir = scene_path / "rgb"
depth_dir = scene_path / "depth"
instance_dir = scene_path / "instances"
In [5]:
# Load images
frame_idx = 209
frame_id = str(frame_idx).zfill(7)
rgb_path = rgb_dir / f"vignette{frame_id}.jpg"
depth_path = depth_dir / f"depth{frame_id}.png"
instance_path = instance_dir / f"instance{frame_id}.png"

rgb = Image.open(rgb_path)
depth = Image.open(depth_path)
instances = Image.open(instance_path)

# Note: Images are rotated to upright for visualization.
# However, the camera calibration is for the original orientation.
rgb_to_plot = rgb.rotate(-90, expand=True)
depth_to_plot = depth.rotate(-90, expand=True)
instances_to_plot = instances.rotate(-90, expand=True)

Part 3: Project Points into Images¶

In [6]:
from projectaria_tools.projects import ase

def transform_3d_points(transform, points):
    N = len(points)
    points_h = np.concatenate([points, np.ones((N, 1))], axis=1)
    transformed_points_h = (transform @ points_h.T).T
    transformed_points = transformed_points_h[:, :-1]
    return transformed_points
    
# Load camera calibration
device = ase.get_ase_rgb_calibration()

# Load the trajectory using read_trajectory_file() 
trajectory_path = scene_path / "trajectory.csv"
trajectory = read_trajectory_file(trajectory_path)

# Load scene point cloud using read_points_file()
points_path = scene_path / "semidense_points.csv.gz"
points_world = read_points_file(points_path)
Loaded trajectory with 350 device poses.
Loaded point cloud with 433426 points.
In [7]:
# Transform the points into the device coordinate frame
T_world_from_device = trajectory["Ts_world_from_device"][frame_idx]
T_device_from_world = np.linalg.inv(T_world_from_device)
points_device = transform_3d_points(T_device_from_world, points_world)

# move from device to camera frame (device is device center but camera is offset)

# Transform the points into the RGB camera coordinate frame
T_device_from_camera = device.get_transform_device_camera().to_matrix()
T_camera_from_device = np.linalg.inv(T_device_from_camera)
points_device = transform_3d_points(T_camera_from_device, points_device)

# Project points into the image
points_image = []

points_device_filtered = []
depths = []
for idx, point_device in enumerate(points_device):
    point_image = device.project(point_device)
    if point_image is not None:
        points_image.append(point_image)
        points_device_filtered.append(point_device)
points_image = np.stack(points_image)

print (len (points_image))

# Overlay projected points onto image
plt.imshow(np.array(rgb))
# plt.scatter(points_image[:, 0], points_image[:, 1], s=0.01, alpha=0.3, c="#FFFFFF")
plt.axis("off")
plt.show()
print (np.array(rgb).shape)

plt.imshow(rgb)
plt.scatter(points_image[:, 0], points_image[:, 1], s=0.01, alpha=0.3, c="#FFFFFF")
plt.axis("off")
plt.show()
56286
No description has been provided for this image
(704, 704, 3)
No description has been provided for this image
In [8]:
points_image = np.array(points_image)
points_device_filtered = np.array(points_device_filtered)
depth_img = np.array(depth)
In [9]:
# make a 3D plot with the depthmap - pull that into 3D, and the camera frame z depth
In [10]:
x = np.arange(depth_img.shape[1])
y = np.arange(depth_img.shape[0])
xx, yy = np.meshgrid(x, y)

x_flat = xx.flatten()
y_flat = yy.flatten()
z_flat = depth_img.flatten()

x_flat_norm = (x_flat-(x_flat.max()/2)) # center
y_flat_norm = (y_flat-(y_flat.max()/2))


plot_traces = [go.Scatter3d(
    x=x_flat_norm,
    y=y_flat_norm,
    z=z_flat,
    mode="markers",
    name="rendered depth map",
    marker={
        "size": 1,
        "opacity": 0.2,
        "color": z_flat,
        "colorscale": "Viridis",
        "colorbar": {"title": "Depth"},
    }
),go.Scatter3d(
    x=points_image[:,0]-(points_image[:,0].max()/2),
    y=points_image[:,1]-(points_image[:,1].max()/2),
    z=points_device_filtered[:,2]*1000,
    mode="markers",
    name="point cloud",
    marker={
        "size": 1,
        "opacity": 0.2,
    }
)
]

fig = go.Figure(data=plot_traces)
fig.update_layout(
    height=600
)

fig.show()